{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "V8-yl-s-WKMG"
      },
      "source": [
        "# Det-AdvProp Tutorial: eval \n",
        "\n",
        "\n",
        "\n",
        "\u003ctable align=\"left\"\u003e\u003ctd\u003e\n",
        "  \u003ca target=\"_blank\"  href=\"https://github.com/google/automl/blob/master/efficientdet/Det-AdvProp-tutorial.ipynb\"\u003e\n",
        "    \u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on github\n",
        "  \u003c/a\u003e\n",
        "\u003c/td\u003e\u003ctd\u003e\n",
        "  \u003ca target=\"_blank\"  href=\"https://colab.sandbox.google.com/github/google/automl/blob/master/efficientdet/det-advprop-tutorial.ipynb\"\u003e\n",
        "    \u003cimg width=32px src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n",
        "\u003c/td\u003e\u003c/table\u003e"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "muwOCNHaq85j"
      },
      "source": [
        "# 0. Install and view graph."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dggLVarNxxvC"
      },
      "source": [
        "## 0.1 Install package and download source code/image.\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hGL97-GXjSUw"
      },
      "outputs": [],
      "source": [
        "%%capture\n",
        "#@title\n",
        "import os\n",
        "import sys\n",
        "import tensorflow.compat.v1 as tf\n",
        "\n",
        "# Download source code.\n",
        "if \"efficientdet\" not in os.getcwd():\n",
        "  !git clone --depth 1 https://github.com/google/automl\n",
        "  os.chdir('automl/efficientdet')\n",
        "  sys.path.append('.')\n",
        "  !pip install -r requirements.txt\n",
        "  !pip install -U 'git+https://github.com/cocodataset/cocoapi.git#subdirectory=PythonAPI'\n",
        "else:\n",
        "  !git pull"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Tow-ic7H3d7i",
        "outputId": "0f9abcf0-9b39-4bf6-9cf5-f98d6d21ff26"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "--2021-05-10 20:07:35--  https://storage.googleapis.com/cloud-tpu-checkpoints/efficientdet/advprop/efficientdet-d1.tar.gz\n",
            "Resolving storage.googleapis.com (storage.googleapis.com)... 74.125.71.128, 64.233.184.128, 74.125.133.128, ...\n",
            "Connecting to storage.googleapis.com (storage.googleapis.com)|74.125.71.128|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 149546361 (143M) [application/octet-stream]\n",
            "Saving to: ‘efficientdet-d1.tar.gz’\n",
            "\n",
            "efficientdet-d1.tar 100%[===================\u003e] 142.62M  39.4MB/s    in 3.6s    \n",
            "\n",
            "2021-05-10 20:07:40 (39.4 MB/s) - ‘efficientdet-d1.tar.gz’ saved [149546361/149546361]\n",
            "\n",
            "Use model in /content/automl/efficientdet/efficientdet-d1\n",
            "--2021-05-10 20:07:42--  https://user-images.githubusercontent.com/11736571/77320690-099af300-6d37-11ea-9d86-24f14dc2d540.png\n",
            "Resolving user-images.githubusercontent.com (user-images.githubusercontent.com)... 185.199.109.133, 185.199.111.133, 185.199.108.133, ...\n",
            "Connecting to user-images.githubusercontent.com (user-images.githubusercontent.com)|185.199.109.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 4080549 (3.9M) [image/png]\n",
            "Saving to: ‘img.png’\n",
            "\n",
            "img.png             100%[===================\u003e]   3.89M  --.-KB/s    in 0.1s    \n",
            "\n",
            "2021-05-10 20:07:42 (38.4 MB/s) - ‘img.png’ saved [4080549/4080549]\n",
            "\n"
          ]
        }
      ],
      "source": [
        "MODEL = 'efficientdet-d1'  #@param\n",
        "\n",
        "def download(m):\n",
        "  if m not in os.listdir():\n",
        "    !wget https://storage.googleapis.com/cloud-tpu-checkpoints/efficientdet/advprop/{m}.tar.gz\n",
        "    !tar zxf {m}.tar.gz\n",
        "  ckpt_path = os.path.join(os.getcwd(), m)\n",
        "  return ckpt_path\n",
        "\n",
        "# Download checkpoint.\n",
        "ckpt_path = download(MODEL)\n",
        "print('Use model in {}'.format(ckpt_path))\n",
        "\n",
        "# Prepare image and visualization settings.\n",
        "image_url =  'https://user-images.githubusercontent.com/11736571/77320690-099af300-6d37-11ea-9d86-24f14dc2d540.png'#@param\n",
        "image_name = 'img.png' #@param\n",
        "!wget {image_url} -O img.png\n",
        "import os\n",
        "img_path = os.path.join(os.getcwd(), 'img.png')\n",
        "\n",
        "min_score_thresh = 0.35  #@param\n",
        "max_boxes_to_draw = 200  #@param\n",
        "line_thickness =   2#@param\n",
        "\n",
        "import PIL\n",
        "# Get the largest of height/width and round to 128.\n",
        "image_size = max(PIL.Image.open(img_path).size)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GvdjcYpUVuQ5"
      },
      "source": [
        "## 0.2 View graph in TensorBoard"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "U2oz3r1LUDzr"
      },
      "outputs": [],
      "source": [
        "!python model_inspect.py --model_name={MODEL} --logdir=logs \u0026\u003e /dev/null\n",
        "%load_ext tensorboard\n",
        "%tensorboard --logdir logs"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RW26DwfirQQN"
      },
      "source": [
        "# 1. COCO evaluation"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cfn_tRFOWKMO"
      },
      "source": [
        "## 1.1 COCO evaluation on validation set."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2s6E8IsVN0pB",
        "outputId": "4e711771-81ca-48c3-c178-ca4228473e6d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "--2021-05-10 19:43:48--  http://images.cocodataset.org/zips/val2017.zip\n",
            "Resolving images.cocodataset.org (images.cocodataset.org)... 52.216.27.28\n",
            "Connecting to images.cocodataset.org (images.cocodataset.org)|52.216.27.28|:80... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 815585330 (778M) [application/zip]\n",
            "Saving to: ‘val2017.zip’\n",
            "\n",
            "val2017.zip         100%[===================\u003e] 777.80M  36.0MB/s    in 22s     \n",
            "\n",
            "2021-05-10 19:44:10 (35.1 MB/s) - ‘val2017.zip’ saved [815585330/815585330]\n",
            "\n",
            "--2021-05-10 19:44:11--  http://images.cocodataset.org/annotations/annotations_trainval2017.zip\n",
            "Resolving images.cocodataset.org (images.cocodataset.org)... 52.217.92.164\n",
            "Connecting to images.cocodataset.org (images.cocodataset.org)|52.217.92.164|:80... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 252907541 (241M) [application/zip]\n",
            "Saving to: ‘annotations_trainval2017.zip’\n",
            "\n",
            "annotations_trainva 100%[===================\u003e] 241.19M  37.0MB/s    in 7.2s    \n",
            "\n",
            "2021-05-10 19:44:18 (33.6 MB/s) - ‘annotations_trainval2017.zip’ saved [252907541/252907541]\n",
            "\n",
            "Archive:  annotations_trainval2017.zip\n",
            "  inflating: annotations/instances_train2017.json  \n",
            "  inflating: annotations/instances_val2017.json  \n",
            "  inflating: annotations/captions_train2017.json  \n",
            "  inflating: annotations/captions_val2017.json  \n",
            "  inflating: annotations/person_keypoints_train2017.json  \n",
            "  inflating: annotations/person_keypoints_val2017.json  \n",
            "2021-05-10 19:44:33.210219: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
            "I0510 19:44:35.059431 140632818317184 create_coco_tfrecord.py:285] writing to output path: tfrecord/val\n",
            "I0510 19:44:35.159756 140632818317184 create_coco_tfrecord.py:237] Building caption index.\n",
            "I0510 19:44:35.166594 140632818317184 create_coco_tfrecord.py:249] 0 images are missing captions.\n",
            "I0510 19:44:36.985283 140632818317184 create_coco_tfrecord.py:323] On image 0 of 5000\n",
            "I0510 19:44:37.145407 140632818317184 create_coco_tfrecord.py:323] On image 100 of 5000\n",
            "I0510 19:44:37.295032 140632818317184 create_coco_tfrecord.py:323] On image 200 of 5000\n",
            "I0510 19:44:37.435169 140632818317184 create_coco_tfrecord.py:323] On image 300 of 5000\n",
            "I0510 19:44:37.580727 140632818317184 create_coco_tfrecord.py:323] On image 400 of 5000\n",
            "I0510 19:44:37.715295 140632818317184 create_coco_tfrecord.py:323] On image 500 of 5000\n",
            "I0510 19:44:37.862454 140632818317184 create_coco_tfrecord.py:323] On image 600 of 5000\n",
            "I0510 19:44:37.993027 140632818317184 create_coco_tfrecord.py:323] On image 700 of 5000\n",
            "I0510 19:44:38.239493 140632818317184 create_coco_tfrecord.py:323] On image 800 of 5000\n",
            "I0510 19:44:38.386704 140632818317184 create_coco_tfrecord.py:323] On image 900 of 5000\n",
            "I0510 19:44:38.555314 140632818317184 create_coco_tfrecord.py:323] On image 1000 of 5000\n",
            "I0510 19:44:38.714288 140632818317184 create_coco_tfrecord.py:323] On image 1100 of 5000\n",
            "I0510 19:44:38.871498 140632818317184 create_coco_tfrecord.py:323] On image 1200 of 5000\n",
            "I0510 19:44:39.004296 140632818317184 create_coco_tfrecord.py:323] On image 1300 of 5000\n",
            "I0510 19:44:39.177606 140632818317184 create_coco_tfrecord.py:323] On image 1400 of 5000\n",
            "I0510 19:44:39.347730 140632818317184 create_coco_tfrecord.py:323] On image 1500 of 5000\n",
            "I0510 19:44:39.513748 140632818317184 create_coco_tfrecord.py:323] On image 1600 of 5000\n",
            "I0510 19:44:39.655339 140632818317184 create_coco_tfrecord.py:323] On image 1700 of 5000\n",
            "I0510 19:44:39.812914 140632818317184 create_coco_tfrecord.py:323] On image 1800 of 5000\n",
            "I0510 19:44:39.939900 140632818317184 create_coco_tfrecord.py:323] On image 1900 of 5000\n",
            "I0510 19:44:40.087029 140632818317184 create_coco_tfrecord.py:323] On image 2000 of 5000\n",
            "I0510 19:44:40.226217 140632818317184 create_coco_tfrecord.py:323] On image 2100 of 5000\n",
            "I0510 19:44:40.365591 140632818317184 create_coco_tfrecord.py:323] On image 2200 of 5000\n",
            "I0510 19:44:40.499703 140632818317184 create_coco_tfrecord.py:323] On image 2300 of 5000\n",
            "I0510 19:44:40.638760 140632818317184 create_coco_tfrecord.py:323] On image 2400 of 5000\n",
            "I0510 19:44:41.050563 140632818317184 create_coco_tfrecord.py:323] On image 2500 of 5000\n",
            "I0510 19:44:41.431067 140632818317184 create_coco_tfrecord.py:323] On image 2600 of 5000\n",
            "I0510 19:44:41.786906 140632818317184 create_coco_tfrecord.py:323] On image 2700 of 5000\n",
            "I0510 19:44:42.212045 140632818317184 create_coco_tfrecord.py:323] On image 2800 of 5000\n",
            "I0510 19:44:42.739339 140632818317184 create_coco_tfrecord.py:323] On image 2900 of 5000\n",
            "I0510 19:44:43.172071 140632818317184 create_coco_tfrecord.py:323] On image 3000 of 5000\n",
            "I0510 19:44:43.651128 140632818317184 create_coco_tfrecord.py:323] On image 3100 of 5000\n",
            "I0510 19:44:44.109982 140632818317184 create_coco_tfrecord.py:323] On image 3200 of 5000\n",
            "I0510 19:44:44.504407 140632818317184 create_coco_tfrecord.py:323] On image 3300 of 5000\n",
            "I0510 19:44:44.679261 140632818317184 create_coco_tfrecord.py:323] On image 3400 of 5000\n",
            "I0510 19:44:44.843348 140632818317184 create_coco_tfrecord.py:323] On image 3500 of 5000\n",
            "I0510 19:44:45.013960 140632818317184 create_coco_tfrecord.py:323] On image 3600 of 5000\n",
            "I0510 19:44:45.209258 140632818317184 create_coco_tfrecord.py:323] On image 3700 of 5000\n",
            "I0510 19:44:45.453207 140632818317184 create_coco_tfrecord.py:323] On image 3800 of 5000\n",
            "I0510 19:44:45.758646 140632818317184 create_coco_tfrecord.py:323] On image 3900 of 5000\n",
            "I0510 19:44:50.063420 140632818317184 create_coco_tfrecord.py:323] On image 4000 of 5000\n",
            "I0510 19:44:50.098973 140632818317184 create_coco_tfrecord.py:323] On image 4100 of 5000\n",
            "I0510 19:44:50.144104 140632818317184 create_coco_tfrecord.py:323] On image 4200 of 5000\n",
            "I0510 19:44:50.187048 140632818317184 create_coco_tfrecord.py:323] On image 4300 of 5000\n",
            "I0510 19:44:50.278038 140632818317184 create_coco_tfrecord.py:323] On image 4400 of 5000\n",
            "I0510 19:44:50.474480 140632818317184 create_coco_tfrecord.py:323] On image 4500 of 5000\n",
            "I0510 19:44:50.887932 140632818317184 create_coco_tfrecord.py:323] On image 4600 of 5000\n",
            "I0510 19:44:51.437325 140632818317184 create_coco_tfrecord.py:323] On image 4700 of 5000\n",
            "I0510 19:44:51.890431 140632818317184 create_coco_tfrecord.py:323] On image 4800 of 5000\n",
            "I0510 19:44:52.091532 140632818317184 create_coco_tfrecord.py:323] On image 4900 of 5000\n",
            "I0510 19:44:52.366421 140632818317184 create_coco_tfrecord.py:335] Finished writing, skipped 0 annotations.\n"
          ]
        }
      ],
      "source": [
        "if 'val2017' not in os.listdir():\n",
        "  !wget http://images.cocodataset.org/zips/val2017.zip\n",
        "  !wget http://images.cocodataset.org/annotations/annotations_trainval2017.zip\n",
        "  !unzip -q val2017.zip\n",
        "  !unzip annotations_trainval2017.zip\n",
        "\n",
        "  !mkdir tfrecord\n",
        "  !PYTHONPATH=\".:$PYTHONPATH\"  python dataset/create_coco_tfrecord.py \\\n",
        "      --image_dir=val2017 \\\n",
        "      --caption_annotations_file=annotations/captions_val2017.json \\\n",
        "      --output_file_prefix=tfrecord/val \\\n",
        "      --num_shards=32"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eLHZUY3jQpZr",
        "outputId": "88f9aff2-708c-45cf-f7d4-c78b21576843"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2021-05-10 20:07:49.603102: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
            "I0510 20:07:52.228707 139834041194368 main.py:264] {'name': 'efficientdet-d1', 'act_type': 'swish', 'image_size': (640, 640), 'target_size': None, 'input_rand_hflip': True, 'jitter_min': 0.1, 'jitter_max': 2.0, 'autoaugment_policy': None, 'grid_mask': False, 'sample_image': None, 'map_freq': 5, 'num_classes': 90, 'seg_num_classes': 3, 'heads': ['object_detection'], 'skip_crowd_during_training': True, 'label_map': None, 'max_instances_per_image': 100, 'regenerate_source_id': False, 'min_level': 3, 'max_level': 7, 'num_scales': 3, 'aspect_ratios': [1.0, 2.0, 0.5], 'anchor_scale': 4.0, 'is_training_bn': True, 'momentum': 0.9, 'optimizer': 'sgd', 'learning_rate': 0.08, 'lr_warmup_init': 0.008, 'lr_warmup_epoch': 1.0, 'first_lr_drop_epoch': 200.0, 'second_lr_drop_epoch': 250.0, 'poly_lr_power': 0.9, 'clip_gradients_norm': 10.0, 'num_epochs': 300, 'data_format': 'channels_last', 'mean_rgb': 0.0, 'stddev_rgb': 1.0, 'scale_range': True, 'label_smoothing': 0.0, 'alpha': 0.25, 'gamma': 1.5, 'delta': 0.1, 'box_loss_weight': 50.0, 'iou_loss_type': None, 'iou_loss_weight': 1.0, 'weight_decay': 4e-05, 'strategy': None, 'mixed_precision': False, 'loss_scale': None, 'model_optimizations': {}, 'box_class_repeats': 3, 'fpn_cell_repeats': 4, 'fpn_num_filters': 88, 'separable_conv': True, 'apply_bn_for_resampling': True, 'conv_after_downsample': False, 'conv_bn_act_pattern': False, 'drop_remainder': True, 'nms_configs': {'method': 'gaussian', 'iou_thresh': None, 'score_thresh': 0.0, 'sigma': None, 'pyfunc': False, 'max_nms_inputs': 0, 'max_output_size': 100}, 'tflite_max_detections': 100, 'fpn_name': None, 'fpn_weight_method': None, 'fpn_config': None, 'survival_prob': None, 'img_summary_steps': None, 'lr_decay_method': 'cosine', 'moving_average_decay': 0.9998, 'ckpt_var_scope': None, 'skip_mismatch': True, 'backbone_name': 'efficientnet-b1', 'backbone_config': None, 'var_freeze_expr': None, 'use_keras_model': True, 'dataset_type': None, 'positives_momentum': None, 'grad_checkpoint': False, 'verbose': 1, 'save_freq': 'epoch', 'model_name': 'efficientdet-d1', 'iterations_per_loop': 100, 'model_dir': '/content/automl/efficientdet/efficientdet-d1', 'num_shards': 8, 'num_examples_per_epoch': 120000, 'backbone_ckpt': '', 'ckpt': None, 'val_json_file': 'annotations/instances_val2017.json', 'testdev_dir': None, 'profile': False, 'mode': 'eval'}\n",
            "INFO:tensorflow:Using config: {'_model_dir': '/content/automl/efficientdet/efficientdet-d1', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': 100, '_save_checkpoints_secs': None, '_session_config': allow_soft_placement: true\n",
            ", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_experimental_max_worker_delay_secs': None, '_session_creation_timeout_secs': 7200, '_checkpoint_save_graph_def': True, '_service': None, '_cluster_spec': ClusterSpec({}), '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}\n",
            "I0510 20:07:52.287737 139834041194368 estimator.py:191] Using config: {'_model_dir': '/content/automl/efficientdet/efficientdet-d1', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': 100, '_save_checkpoints_secs': None, '_session_config': allow_soft_placement: true\n",
            ", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_experimental_max_worker_delay_secs': None, '_session_creation_timeout_secs': 7200, '_checkpoint_save_graph_def': True, '_service': None, '_cluster_spec': ClusterSpec({}), '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}\n",
            "INFO:tensorflow:Using config: {'_model_dir': '/content/automl/efficientdet/efficientdet-d1', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': 100, '_save_checkpoints_secs': None, '_session_config': allow_soft_placement: true\n",
            ", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_experimental_max_worker_delay_secs': None, '_session_creation_timeout_secs': 7200, '_checkpoint_save_graph_def': True, '_service': None, '_cluster_spec': ClusterSpec({}), '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}\n",
            "I0510 20:07:52.288912 139834041194368 estimator.py:191] Using config: {'_model_dir': '/content/automl/efficientdet/efficientdet-d1', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': 100, '_save_checkpoints_secs': None, '_session_config': allow_soft_placement: true\n",
            ", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_experimental_max_worker_delay_secs': None, '_session_creation_timeout_secs': 7200, '_checkpoint_save_graph_def': True, '_service': None, '_cluster_spec': ClusterSpec({}), '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}\n",
            "INFO:tensorflow:Waiting for new checkpoint at /content/automl/efficientdet/efficientdet-d1\n",
            "I0510 20:07:52.289403 139834041194368 checkpoint_utils.py:139] Waiting for new checkpoint at /content/automl/efficientdet/efficientdet-d1\n",
            "INFO:tensorflow:Found new checkpoint at /content/automl/efficientdet/efficientdet-d1/model\n",
            "I0510 20:07:52.292483 139834041194368 checkpoint_utils.py:148] Found new checkpoint at /content/automl/efficientdet/efficientdet-d1/model\n",
            "I0510 20:07:52.292706 139834041194368 main.py:344] Starting to evaluate.\n",
            "2021-05-10 20:07:52.489416: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set\n",
            "2021-05-10 20:07:52.490577: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1\n",
            "2021-05-10 20:07:52.508703: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:941] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
            "2021-05-10 20:07:52.509616: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: \n",
            "pciBusID: 0000:00:04.0 name: Tesla P100-PCIE-16GB computeCapability: 6.0\n",
            "coreClock: 1.3285GHz coreCount: 56 deviceMemorySize: 15.90GiB deviceMemoryBandwidth: 681.88GiB/s\n",
            "2021-05-10 20:07:52.509660: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
            "2021-05-10 20:07:52.512220: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11\n",
            "2021-05-10 20:07:52.512295: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11\n",
            "2021-05-10 20:07:52.514123: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10\n",
            "2021-05-10 20:07:52.514558: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10\n",
            "2021-05-10 20:07:52.516516: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10\n",
            "2021-05-10 20:07:52.517094: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11\n",
            "2021-05-10 20:07:52.517346: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8\n",
            "2021-05-10 20:07:52.517450: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:941] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
            "2021-05-10 20:07:52.518379: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:941] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
            "2021-05-10 20:07:52.519223: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0\n",
            "INFO:tensorflow:Calling model_fn.\n",
            "I0510 20:07:53.088464 139834041194368 estimator.py:1162] Calling model_fn.\n",
            "I0510 20:07:53.093623 139834041194368 efficientnet_builder.py:215] global_params= GlobalParams(batch_norm_momentum=0.99, batch_norm_epsilon=0.001, dropout_rate=0.2, data_format='channels_last', num_classes=1000, width_coefficient=1.0, depth_coefficient=1.1, depth_divisor=8, min_depth=None, survival_prob=0.8, relu_fn=functools.partial(\u003cfunction activation_fn at 0x7f2d4fb79cb0\u003e, act_type='swish'), batch_norm=\u003cclass 'utils.BatchNormalization'\u003e, use_se=True, local_pooling=None, condconv_num_experts=None, clip_projection_output=False, blocks_args=['r1_k3_s11_e1_i32_o16_se0.25', 'r2_k3_s22_e6_i16_o24_se0.25', 'r2_k5_s22_e6_i24_o40_se0.25', 'r3_k3_s22_e6_i40_o80_se0.25', 'r3_k5_s11_e6_i80_o112_se0.25', 'r4_k5_s22_e6_i112_o192_se0.25', 'r1_k3_s11_e6_i192_o320_se0.25'], fix_head_stem=None, grad_checkpoint=False)\n",
            "I0510 20:07:53.595246 139834041194368 efficientdet_keras.py:760] fnode 0 : {'feat_level': 6, 'inputs_offsets': [3, 4]}\n",
            "I0510 20:07:53.596441 139834041194368 efficientdet_keras.py:760] fnode 1 : {'feat_level': 5, 'inputs_offsets': [2, 5]}\n",
            "I0510 20:07:53.597646 139834041194368 efficientdet_keras.py:760] fnode 2 : {'feat_level': 4, 'inputs_offsets': [1, 6]}\n",
            "I0510 20:07:53.598873 139834041194368 efficientdet_keras.py:760] fnode 3 : {'feat_level': 3, 'inputs_offsets': [0, 7]}\n",
            "I0510 20:07:53.600070 139834041194368 efficientdet_keras.py:760] fnode 4 : {'feat_level': 4, 'inputs_offsets': [1, 7, 8]}\n",
            "I0510 20:07:53.601099 139834041194368 efficientdet_keras.py:760] fnode 5 : {'feat_level': 5, 'inputs_offsets': [2, 6, 9]}\n",
            "I0510 20:07:53.602200 139834041194368 efficientdet_keras.py:760] fnode 6 : {'feat_level': 6, 'inputs_offsets': [3, 5, 10]}\n",
            "I0510 20:07:53.603379 139834041194368 efficientdet_keras.py:760] fnode 7 : {'feat_level': 7, 'inputs_offsets': [4, 11]}\n",
            "I0510 20:07:53.606173 139834041194368 efficientdet_keras.py:760] fnode 0 : {'feat_level': 6, 'inputs_offsets': [3, 4]}\n",
            "I0510 20:07:53.607471 139834041194368 efficientdet_keras.py:760] fnode 1 : {'feat_level': 5, 'inputs_offsets': [2, 5]}\n",
            "I0510 20:07:53.608979 139834041194368 efficientdet_keras.py:760] fnode 2 : {'feat_level': 4, 'inputs_offsets': [1, 6]}\n",
            "I0510 20:07:53.610496 139834041194368 efficientdet_keras.py:760] fnode 3 : {'feat_level': 3, 'inputs_offsets': [0, 7]}\n",
            "I0510 20:07:53.611800 139834041194368 efficientdet_keras.py:760] fnode 4 : {'feat_level': 4, 'inputs_offsets': [1, 7, 8]}\n",
            "I0510 20:07:53.613338 139834041194368 efficientdet_keras.py:760] fnode 5 : {'feat_level': 5, 'inputs_offsets': [2, 6, 9]}\n",
            "I0510 20:07:53.615097 139834041194368 efficientdet_keras.py:760] fnode 6 : {'feat_level': 6, 'inputs_offsets': [3, 5, 10]}\n",
            "I0510 20:07:53.617069 139834041194368 efficientdet_keras.py:760] fnode 7 : {'feat_level': 7, 'inputs_offsets': [4, 11]}\n",
            "I0510 20:07:53.619574 139834041194368 efficientdet_keras.py:760] fnode 0 : {'feat_level': 6, 'inputs_offsets': [3, 4]}\n",
            "I0510 20:07:53.620916 139834041194368 efficientdet_keras.py:760] fnode 1 : {'feat_level': 5, 'inputs_offsets': [2, 5]}\n",
            "I0510 20:07:53.622064 139834041194368 efficientdet_keras.py:760] fnode 2 : {'feat_level': 4, 'inputs_offsets': [1, 6]}\n",
            "I0510 20:07:53.623369 139834041194368 efficientdet_keras.py:760] fnode 3 : {'feat_level': 3, 'inputs_offsets': [0, 7]}\n",
            "I0510 20:07:53.624463 139834041194368 efficientdet_keras.py:760] fnode 4 : {'feat_level': 4, 'inputs_offsets': [1, 7, 8]}\n",
            "I0510 20:07:53.625886 139834041194368 efficientdet_keras.py:760] fnode 5 : {'feat_level': 5, 'inputs_offsets': [2, 6, 9]}\n",
            "I0510 20:07:53.627297 139834041194368 efficientdet_keras.py:760] fnode 6 : {'feat_level': 6, 'inputs_offsets': [3, 5, 10]}\n",
            "I0510 20:07:53.628497 139834041194368 efficientdet_keras.py:760] fnode 7 : {'feat_level': 7, 'inputs_offsets': [4, 11]}\n",
            "I0510 20:07:53.630418 139834041194368 efficientdet_keras.py:760] fnode 0 : {'feat_level': 6, 'inputs_offsets': [3, 4]}\n",
            "I0510 20:07:53.631624 139834041194368 efficientdet_keras.py:760] fnode 1 : {'feat_level': 5, 'inputs_offsets': [2, 5]}\n",
            "I0510 20:07:53.632795 139834041194368 efficientdet_keras.py:760] fnode 2 : {'feat_level': 4, 'inputs_offsets': [1, 6]}\n",
            "I0510 20:07:53.633984 139834041194368 efficientdet_keras.py:760] fnode 3 : {'feat_level': 3, 'inputs_offsets': [0, 7]}\n",
            "I0510 20:07:53.635079 139834041194368 efficientdet_keras.py:760] fnode 4 : {'feat_level': 4, 'inputs_offsets': [1, 7, 8]}\n",
            "I0510 20:07:53.636193 139834041194368 efficientdet_keras.py:760] fnode 5 : {'feat_level': 5, 'inputs_offsets': [2, 6, 9]}\n",
            "I0510 20:07:53.637408 139834041194368 efficientdet_keras.py:760] fnode 6 : {'feat_level': 6, 'inputs_offsets': [3, 5, 10]}\n",
            "I0510 20:07:53.638536 139834041194368 efficientdet_keras.py:760] fnode 7 : {'feat_level': 7, 'inputs_offsets': [4, 11]}\n",
            "I0510 20:07:53.750490 139834041194368 efficientnet_model.py:735] Built stem stem : (1, 320, 320, 32)\n",
            "I0510 20:07:53.750730 139834041194368 efficientnet_model.py:756] block_0 survival_prob: 1.0\n",
            "I0510 20:07:53.751127 139834041194368 efficientnet_model.py:374] Block blocks_0 input shape: (1, 320, 320, 32)\n",
            "I0510 20:07:53.780265 139834041194368 efficientnet_model.py:393] DWConv shape: (1, 320, 320, 32)\n",
            "I0510 20:07:53.809039 139834041194368 efficientnet_model.py:195] Built SE se : (1, 1, 1, 32)\n",
            "I0510 20:07:53.836266 139834041194368 efficientnet_model.py:414] Project shape: (1, 320, 320, 16)\n",
            "I0510 20:07:53.836619 139834041194368 efficientnet_model.py:756] block_1 survival_prob: 0.991304347826087\n",
            "I0510 20:07:53.837053 139834041194368 efficientnet_model.py:374] Block blocks_1 input shape: (1, 320, 320, 16)\n",
            "I0510 20:07:53.871055 139834041194368 efficientnet_model.py:393] DWConv shape: (1, 320, 320, 16)\n",
            "I0510 20:07:53.900743 139834041194368 efficientnet_model.py:195] Built SE se : (1, 1, 1, 16)\n",
            "I0510 20:07:53.930487 139834041194368 efficientnet_model.py:414] Project shape: (1, 320, 320, 16)\n",
            "I0510 20:07:53.930959 139834041194368 efficientnet_model.py:756] block_2 survival_prob: 0.9826086956521739\n",
            "I0510 20:07:53.931439 139834041194368 efficientnet_model.py:374] Block blocks_2 input shape: (1, 320, 320, 16)\n",
            "I0510 20:07:53.961296 139834041194368 efficientnet_model.py:390] Expand shape: (1, 320, 320, 96)\n",
            "I0510 20:07:53.993832 139834041194368 efficientnet_model.py:393] DWConv shape: (1, 160, 160, 96)\n",
            "I0510 20:07:54.025872 139834041194368 efficientnet_model.py:195] Built SE se : (1, 1, 1, 96)\n",
            "I0510 20:07:54.059447 139834041194368 efficientnet_model.py:414] Project shape: (1, 160, 160, 24)\n",
            "I0510 20:07:54.059780 139834041194368 efficientnet_model.py:756] block_3 survival_prob: 0.9739130434782609\n",
            "I0510 20:07:54.060328 139834041194368 efficientnet_model.py:374] Block blocks_3 input shape: (1, 160, 160, 24)\n",
            "I0510 20:07:54.088966 139834041194368 efficientnet_model.py:390] Expand shape: (1, 160, 160, 144)\n",
            "I0510 20:07:54.121675 139834041194368 efficientnet_model.py:393] DWConv shape: (1, 160, 160, 144)\n",
            "I0510 20:07:54.150545 139834041194368 efficientnet_model.py:195] Built SE se : (1, 1, 1, 144)\n",
            "I0510 20:07:54.182002 139834041194368 efficientnet_model.py:414] Project shape: (1, 160, 160, 24)\n",
            "I0510 20:07:54.182313 139834041194368 efficientnet_model.py:756] block_4 survival_prob: 0.9652173913043478\n",
            "I0510 20:07:54.182894 139834041194368 efficientnet_model.py:374] Block blocks_4 input shape: (1, 160, 160, 24)\n",
            "I0510 20:07:54.211359 139834041194368 efficientnet_model.py:390] Expand shape: (1, 160, 160, 144)\n",
            "I0510 20:07:54.241073 139834041194368 efficientnet_model.py:393] DWConv shape: (1, 160, 160, 144)\n",
            "I0510 20:07:54.271023 139834041194368 efficientnet_model.py:195] Built SE se : (1, 1, 1, 144)\n",
            "I0510 20:07:54.299235 139834041194368 efficientnet_model.py:414] Project shape: (1, 160, 160, 24)\n",
            "I0510 20:07:54.299630 139834041194368 efficientnet_model.py:756] block_5 survival_prob: 0.9565217391304348\n",
            "I0510 20:07:54.300070 139834041194368 efficientnet_model.py:374] Block blocks_5 input shape: (1, 160, 160, 24)\n",
            "I0510 20:07:54.328530 139834041194368 efficientnet_model.py:390] Expand shape: (1, 160, 160, 144)\n",
            "I0510 20:07:54.358716 139834041194368 efficientnet_model.py:393] DWConv shape: (1, 80, 80, 144)\n",
            "I0510 20:07:54.389357 139834041194368 efficientnet_model.py:195] Built SE se : (1, 1, 1, 144)\n",
            "I0510 20:07:54.418314 139834041194368 efficientnet_model.py:414] Project shape: (1, 80, 80, 40)\n",
            "I0510 20:07:54.418715 139834041194368 efficientnet_model.py:756] block_6 survival_prob: 0.9478260869565217\n",
            "I0510 20:07:54.419190 139834041194368 efficientnet_model.py:374] Block blocks_6 input shape: (1, 80, 80, 40)\n",
            "I0510 20:07:54.451548 139834041194368 efficientnet_model.py:390] Expand shape: (1, 80, 80, 240)\n",
            "I0510 20:07:54.486734 139834041194368 efficientnet_model.py:393] DWConv shape: (1, 80, 80, 240)\n",
            "I0510 20:07:54.516430 139834041194368 efficientnet_model.py:195] Built SE se : (1, 1, 1, 240)\n",
            "I0510 20:07:54.543974 139834041194368 efficientnet_model.py:414] Project shape: (1, 80, 80, 40)\n",
            "I0510 20:07:54.544296 139834041194368 efficientnet_model.py:756] block_7 survival_prob: 0.9391304347826087\n",
            "I0510 20:07:54.544718 139834041194368 efficientnet_model.py:374] Block blocks_7 input shape: (1, 80, 80, 40)\n",
            "I0510 20:07:54.573758 139834041194368 efficientnet_model.py:390] Expand shape: (1, 80, 80, 240)\n",
            "I0510 20:07:54.607336 139834041194368 efficientnet_model.py:393] DWConv shape: (1, 80, 80, 240)\n",
            "I0510 20:07:54.636443 139834041194368 efficientnet_model.py:195] Built SE se : (1, 1, 1, 240)\n",
            "I0510 20:07:54.663759 139834041194368 efficientnet_model.py:414] Project shape: (1, 80, 80, 40)\n",
            "I0510 20:07:54.664189 139834041194368 efficientnet_model.py:756] block_8 survival_prob: 0.9304347826086956\n",
            "I0510 20:07:54.664561 139834041194368 efficientnet_model.py:374] Block blocks_8 input shape: (1, 80, 80, 40)\n",
            "I0510 20:07:54.696891 139834041194368 efficientnet_model.py:390] Expand shape: (1, 80, 80, 240)\n",
            "I0510 20:07:54.726265 139834041194368 efficientnet_model.py:393] DWConv shape: (1, 40, 40, 240)\n",
            "I0510 20:07:54.755805 139834041194368 efficientnet_model.py:195] Built SE se : (1, 1, 1, 240)\n",
            "I0510 20:07:54.786793 139834041194368 efficientnet_model.py:414] Project shape: (1, 40, 40, 80)\n",
            "I0510 20:07:54.787131 139834041194368 efficientnet_model.py:756] block_9 survival_prob: 0.9217391304347826\n",
            "I0510 20:07:54.787655 139834041194368 efficientnet_model.py:374] Block blocks_9 input shape: (1, 40, 40, 80)\n",
            "I0510 20:07:54.815595 139834041194368 efficientnet_model.py:390] Expand shape: (1, 40, 40, 480)\n",
            "I0510 20:07:54.844414 139834041194368 efficientnet_model.py:393] DWConv shape: (1, 40, 40, 480)\n",
            "I0510 20:07:54.874174 139834041194368 efficientnet_model.py:195] Built SE se : (1, 1, 1, 480)\n",
            "I0510 20:07:54.902137 139834041194368 efficientnet_model.py:414] Project shape: (1, 40, 40, 80)\n",
            "I0510 20:07:54.902514 139834041194368 efficientnet_model.py:756] block_10 survival_prob: 0.9130434782608696\n",
            "I0510 20:07:54.902967 139834041194368 efficientnet_model.py:374] Block blocks_10 input shape: (1, 40, 40, 80)\n",
            "I0510 20:07:54.931709 139834041194368 efficientnet_model.py:390] Expand shape: (1, 40, 40, 480)\n",
            "I0510 20:07:54.962241 139834041194368 efficientnet_model.py:393] DWConv shape: (1, 40, 40, 480)\n",
            "I0510 20:07:54.993660 139834041194368 efficientnet_model.py:195] Built SE se : (1, 1, 1, 480)\n",
            "I0510 20:07:55.021482 139834041194368 efficientnet_model.py:414] Project shape: (1, 40, 40, 80)\n",
            "I0510 20:07:55.021827 139834041194368 efficientnet_model.py:756] block_11 survival_prob: 0.9043478260869565\n",
            "I0510 20:07:55.022259 139834041194368 efficientnet_model.py:374] Block blocks_11 input shape: (1, 40, 40, 80)\n",
            "I0510 20:07:55.057896 139834041194368 efficientnet_model.py:390] Expand shape: (1, 40, 40, 480)\n",
            "I0510 20:07:55.091666 139834041194368 efficientnet_model.py:393] DWConv shape: (1, 40, 40, 480)\n",
            "I0510 20:07:55.121704 139834041194368 efficientnet_model.py:195] Built SE se : (1, 1, 1, 480)\n",
            "I0510 20:07:55.149158 139834041194368 efficientnet_model.py:414] Project shape: (1, 40, 40, 80)\n",
            "I0510 20:07:55.149526 139834041194368 efficientnet_model.py:756] block_12 survival_prob: 0.8956521739130435\n",
            "I0510 20:07:55.150015 139834041194368 efficientnet_model.py:374] Block blocks_12 input shape: (1, 40, 40, 80)\n",
            "I0510 20:07:55.178953 139834041194368 efficientnet_model.py:390] Expand shape: (1, 40, 40, 480)\n",
            "I0510 20:07:55.209172 139834041194368 efficientnet_model.py:393] DWConv shape: (1, 40, 40, 480)\n",
            "I0510 20:07:55.238858 139834041194368 efficientnet_model.py:195] Built SE se : (1, 1, 1, 480)\n",
            "I0510 20:07:55.267069 139834041194368 efficientnet_model.py:414] Project shape: (1, 40, 40, 112)\n",
            "I0510 20:07:55.267507 139834041194368 efficientnet_model.py:756] block_13 survival_prob: 0.8869565217391304\n",
            "I0510 20:07:55.268048 139834041194368 efficientnet_model.py:374] Block blocks_13 input shape: (1, 40, 40, 112)\n",
            "I0510 20:07:55.296053 139834041194368 efficientnet_model.py:390] Expand shape: (1, 40, 40, 672)\n",
            "I0510 20:07:55.331140 139834041194368 efficientnet_model.py:393] DWConv shape: (1, 40, 40, 672)\n",
            "I0510 20:07:55.360097 139834041194368 efficientnet_model.py:195] Built SE se : (1, 1, 1, 672)\n",
            "I0510 20:07:55.391819 139834041194368 efficientnet_model.py:414] Project shape: (1, 40, 40, 112)\n",
            "I0510 20:07:55.392213 139834041194368 efficientnet_model.py:756] block_14 survival_prob: 0.8782608695652174\n",
            "I0510 20:07:55.392651 139834041194368 efficientnet_model.py:374] Block blocks_14 input shape: (1, 40, 40, 112)\n",
            "I0510 20:07:55.422633 139834041194368 efficientnet_model.py:390] Expand shape: (1, 40, 40, 672)\n",
            "I0510 20:07:55.452295 139834041194368 efficientnet_model.py:393] DWConv shape: (1, 40, 40, 672)\n",
            "I0510 20:07:55.482393 139834041194368 efficientnet_model.py:195] Built SE se : (1, 1, 1, 672)\n",
            "I0510 20:07:55.512572 139834041194368 efficientnet_model.py:414] Project shape: (1, 40, 40, 112)\n",
            "I0510 20:07:55.512948 139834041194368 efficientnet_model.py:756] block_15 survival_prob: 0.8695652173913044\n",
            "I0510 20:07:55.513380 139834041194368 efficientnet_model.py:374] Block blocks_15 input shape: (1, 40, 40, 112)\n",
            "I0510 20:07:55.542927 139834041194368 efficientnet_model.py:390] Expand shape: (1, 40, 40, 672)\n",
            "I0510 20:07:55.574158 139834041194368 efficientnet_model.py:393] DWConv shape: (1, 40, 40, 672)\n",
            "I0510 20:07:55.605713 139834041194368 efficientnet_model.py:195] Built SE se : (1, 1, 1, 672)\n",
            "I0510 20:07:55.633415 139834041194368 efficientnet_model.py:414] Project shape: (1, 40, 40, 112)\n",
            "I0510 20:07:55.633908 139834041194368 efficientnet_model.py:756] block_16 survival_prob: 0.8608695652173913\n",
            "I0510 20:07:55.634328 139834041194368 efficientnet_model.py:374] Block blocks_16 input shape: (1, 40, 40, 112)\n",
            "I0510 20:07:55.663101 139834041194368 efficientnet_model.py:390] Expand shape: (1, 40, 40, 672)\n",
            "I0510 20:07:55.696253 139834041194368 efficientnet_model.py:393] DWConv shape: (1, 20, 20, 672)\n",
            "I0510 20:07:55.728366 139834041194368 efficientnet_model.py:195] Built SE se : (1, 1, 1, 672)\n",
            "I0510 20:07:55.756615 139834041194368 efficientnet_model.py:414] Project shape: (1, 20, 20, 192)\n",
            "I0510 20:07:55.756961 139834041194368 efficientnet_model.py:756] block_17 survival_prob: 0.8521739130434782\n",
            "I0510 20:07:55.757386 139834041194368 efficientnet_model.py:374] Block blocks_17 input shape: (1, 20, 20, 192)\n",
            "I0510 20:07:55.792434 139834041194368 efficientnet_model.py:390] Expand shape: (1, 20, 20, 1152)\n",
            "I0510 20:07:55.829316 139834041194368 efficientnet_model.py:393] DWConv shape: (1, 20, 20, 1152)\n",
            "I0510 20:07:55.860946 139834041194368 efficientnet_model.py:195] Built SE se : (1, 1, 1, 1152)\n",
            "I0510 20:07:55.888971 139834041194368 efficientnet_model.py:414] Project shape: (1, 20, 20, 192)\n",
            "I0510 20:07:55.889437 139834041194368 efficientnet_model.py:756] block_18 survival_prob: 0.8434782608695652\n",
            "I0510 20:07:55.889867 139834041194368 efficientnet_model.py:374] Block blocks_18 input shape: (1, 20, 20, 192)\n",
            "I0510 20:07:55.924473 139834041194368 efficientnet_model.py:390] Expand shape: (1, 20, 20, 1152)\n",
            "I0510 20:07:55.968310 139834041194368 efficientnet_model.py:393] DWConv shape: (1, 20, 20, 1152)\n",
            "I0510 20:07:56.009804 139834041194368 efficientnet_model.py:195] Built SE se : (1, 1, 1, 1152)\n",
            "I0510 20:07:56.041101 139834041194368 efficientnet_model.py:414] Project shape: (1, 20, 20, 192)\n",
            "I0510 20:07:56.041589 139834041194368 efficientnet_model.py:756] block_19 survival_prob: 0.8347826086956522\n",
            "I0510 20:07:56.042091 139834041194368 efficientnet_model.py:374] Block blocks_19 input shape: (1, 20, 20, 192)\n",
            "I0510 20:07:56.093295 139834041194368 efficientnet_model.py:390] Expand shape: (1, 20, 20, 1152)\n",
            "I0510 20:07:56.136825 139834041194368 efficientnet_model.py:393] DWConv shape: (1, 20, 20, 1152)\n",
            "I0510 20:07:56.168875 139834041194368 efficientnet_model.py:195] Built SE se : (1, 1, 1, 1152)\n",
            "I0510 20:07:56.195683 139834041194368 efficientnet_model.py:414] Project shape: (1, 20, 20, 192)\n",
            "I0510 20:07:56.196047 139834041194368 efficientnet_model.py:756] block_20 survival_prob: 0.8260869565217391\n",
            "I0510 20:07:56.196481 139834041194368 efficientnet_model.py:374] Block blocks_20 input shape: (1, 20, 20, 192)\n",
            "I0510 20:07:56.234993 139834041194368 efficientnet_model.py:390] Expand shape: (1, 20, 20, 1152)\n",
            "I0510 20:07:56.272101 139834041194368 efficientnet_model.py:393] DWConv shape: (1, 20, 20, 1152)\n",
            "I0510 20:07:56.306418 139834041194368 efficientnet_model.py:195] Built SE se : (1, 1, 1, 1152)\n",
            "I0510 20:07:56.335123 139834041194368 efficientnet_model.py:414] Project shape: (1, 20, 20, 192)\n",
            "I0510 20:07:56.335595 139834041194368 efficientnet_model.py:756] block_21 survival_prob: 0.8173913043478261\n",
            "I0510 20:07:56.336132 139834041194368 efficientnet_model.py:374] Block blocks_21 input shape: (1, 20, 20, 192)\n",
            "I0510 20:07:56.374043 139834041194368 efficientnet_model.py:390] Expand shape: (1, 20, 20, 1152)\n",
            "I0510 20:07:56.412933 139834041194368 efficientnet_model.py:393] DWConv shape: (1, 20, 20, 1152)\n",
            "I0510 20:07:56.446010 139834041194368 efficientnet_model.py:195] Built SE se : (1, 1, 1, 1152)\n",
            "I0510 20:07:56.475004 139834041194368 efficientnet_model.py:414] Project shape: (1, 20, 20, 320)\n",
            "I0510 20:07:56.475410 139834041194368 efficientnet_model.py:756] block_22 survival_prob: 0.808695652173913\n",
            "I0510 20:07:56.476164 139834041194368 efficientnet_model.py:374] Block blocks_22 input shape: (1, 20, 20, 320)\n",
            "I0510 20:07:56.514158 139834041194368 efficientnet_model.py:390] Expand shape: (1, 20, 20, 1920)\n",
            "I0510 20:07:56.550471 139834041194368 efficientnet_model.py:393] DWConv shape: (1, 20, 20, 1920)\n",
            "I0510 20:07:56.586025 139834041194368 efficientnet_model.py:195] Built SE se : (1, 1, 1, 1920)\n",
            "I0510 20:07:56.615339 139834041194368 efficientnet_model.py:414] Project shape: (1, 20, 20, 320)\n",
            "I0510 20:08:00.628787 139834041194368 det_model_fn.py:81] LR schedule method: cosine\n",
            "I0510 20:08:01.077699 139834041194368 postprocess.py:90] use max_nms_inputs for pre-nms topk.\n",
            "I0510 20:08:02.011834 139834041194368 det_model_fn.py:476] Eval val with groudtruths annotations/instances_val2017.json.\n",
            "I0510 20:08:02.068171 139834041194368 det_model_fn.py:553] Load EMA vars with ema_decay=0.999800\n",
            "INFO:tensorflow:Done calling model_fn.\n",
            "I0510 20:08:03.211773 139834041194368 estimator.py:1164] Done calling model_fn.\n",
            "INFO:tensorflow:Starting evaluation at 2021-05-10T20:08:03Z\n",
            "I0510 20:08:03.234877 139834041194368 evaluation.py:255] Starting evaluation at 2021-05-10T20:08:03Z\n",
            "INFO:tensorflow:Graph was finalized.\n",
            "I0510 20:08:04.095394 139834041194368 monitored_session.py:246] Graph was finalized.\n",
            "2021-05-10 20:08:04.096106: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set\n",
            "2021-05-10 20:08:04.096396: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:941] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
            "2021-05-10 20:08:04.097427: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: \n",
            "pciBusID: 0000:00:04.0 name: Tesla P100-PCIE-16GB computeCapability: 6.0\n",
            "coreClock: 1.3285GHz coreCount: 56 deviceMemorySize: 15.90GiB deviceMemoryBandwidth: 681.88GiB/s\n",
            "2021-05-10 20:08:04.097507: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
            "2021-05-10 20:08:04.097573: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11\n",
            "2021-05-10 20:08:04.097644: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11\n",
            "2021-05-10 20:08:04.097691: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10\n",
            "2021-05-10 20:08:04.097739: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10\n",
            "2021-05-10 20:08:04.097786: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10\n",
            "2021-05-10 20:08:04.097891: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11\n",
            "2021-05-10 20:08:04.097938: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8\n",
            "2021-05-10 20:08:04.098070: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:941] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
            "2021-05-10 20:08:04.099073: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:941] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
            "2021-05-10 20:08:04.099983: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0\n",
            "2021-05-10 20:08:04.100061: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
            "2021-05-10 20:08:04.802699: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:\n",
            "2021-05-10 20:08:04.802772: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267]      0 \n",
            "2021-05-10 20:08:04.802798: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0:   N \n",
            "2021-05-10 20:08:04.803063: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:941] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
            "2021-05-10 20:08:04.804184: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:941] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
            "2021-05-10 20:08:04.805216: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:941] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
            "2021-05-10 20:08:04.806026: W tensorflow/core/common_runtime/gpu/gpu_bfc_allocator.cc:39] Overriding allow_growth setting because the TF_FORCE_GPU_ALLOW_GROWTH environment variable is set. Original config value was 0.\n",
            "2021-05-10 20:08:04.806089: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 14975 MB memory) -\u003e physical GPU (device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:04.0, compute capability: 6.0)\n",
            "INFO:tensorflow:Restoring parameters from /content/automl/efficientdet/efficientdet-d1/model\n",
            "I0510 20:08:04.806891 139834041194368 saver.py:1292] Restoring parameters from /content/automl/efficientdet/efficientdet-d1/model\n",
            "2021-05-10 20:08:04.944665: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:196] None of the MLIR optimization passes are enabled (registered 0 passes)\n",
            "2021-05-10 20:08:05.123288: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2299995000 Hz\n",
            "INFO:tensorflow:Running local_init_op.\n",
            "I0510 20:08:06.682386 139834041194368 session_manager.py:505] Running local_init_op.\n",
            "INFO:tensorflow:Done running local_init_op.\n",
            "I0510 20:08:06.769455 139834041194368 session_manager.py:508] Done running local_init_op.\n",
            "2021-05-10 20:08:07.729877: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2)\n",
            "2021-05-10 20:08:10.942540: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8\n",
            "2021-05-10 20:08:12.145170: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11\n",
            "2021-05-10 20:08:12.421326: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11\n",
            "INFO:tensorflow:Evaluation [500/5000]\n",
            "I0510 20:09:43.242785 139834041194368 evaluation.py:167] Evaluation [500/5000]\n",
            "INFO:tensorflow:Evaluation [1000/5000]\n",
            "I0510 20:11:13.138355 139834041194368 evaluation.py:167] Evaluation [1000/5000]\n",
            "INFO:tensorflow:Evaluation [1500/5000]\n",
            "I0510 20:12:42.090077 139834041194368 evaluation.py:167] Evaluation [1500/5000]\n",
            "INFO:tensorflow:Evaluation [2000/5000]\n",
            "I0510 20:14:10.714957 139834041194368 evaluation.py:167] Evaluation [2000/5000]\n",
            "INFO:tensorflow:Evaluation [2500/5000]\n",
            "I0510 20:15:39.312114 139834041194368 evaluation.py:167] Evaluation [2500/5000]\n",
            "INFO:tensorflow:Evaluation [3000/5000]\n",
            "I0510 20:17:08.327113 139834041194368 evaluation.py:167] Evaluation [3000/5000]\n",
            "INFO:tensorflow:Evaluation [3500/5000]\n",
            "I0510 20:18:37.902203 139834041194368 evaluation.py:167] Evaluation [3500/5000]\n",
            "INFO:tensorflow:Evaluation [4000/5000]\n",
            "I0510 20:20:06.601424 139834041194368 evaluation.py:167] Evaluation [4000/5000]\n",
            "INFO:tensorflow:Evaluation [4500/5000]\n",
            "I0510 20:21:35.209655 139834041194368 evaluation.py:167] Evaluation [4500/5000]\n",
            "INFO:tensorflow:Evaluation [5000/5000]\n",
            "I0510 20:23:04.230208 139834041194368 evaluation.py:167] Evaluation [5000/5000]\n",
            "loading annotations into memory...\n",
            "Done (t=0.76s)\n",
            "creating index...\n",
            "index created!\n",
            "Loading and preparing results...\n",
            "Converting ndarray to lists...\n",
            "(500000, 7)\n",
            "0/500000\n",
            "DONE (t=4.48s)\n",
            "creating index...\n",
            "index created!\n",
            "Running per image evaluation...\n",
            "Evaluate annotation type *bbox*\n",
            "DONE (t=90.24s).\n",
            "Accumulating evaluation results...\n",
            "DONE (t=14.09s).\n",
            " Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.408\n",
            " Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=100 ] = 0.599\n",
            " Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=100 ] = 0.440\n",
            " Average Precision  (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.214\n",
            " Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.463\n",
            " Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.591\n",
            " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=  1 ] = 0.328\n",
            " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 10 ] = 0.520\n",
            " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.551\n",
            " Average Recall     (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.326\n",
            " Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.624\n",
            " Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.748\n",
            "INFO:tensorflow:Inference Time : 1013.67302s\n",
            "I0510 20:24:56.908151 139834041194368 evaluation.py:273] Inference Time : 1013.67302s\n",
            "INFO:tensorflow:Finished evaluation at 2021-05-10-20:24:56\n",
            "I0510 20:24:56.908450 139834041194368 evaluation.py:276] Finished evaluation at 2021-05-10-20:24:56\n",
            "INFO:tensorflow:Saving dict for global step 0: AP = 0.40841353, AP50 = 0.5991094, AP75 = 0.4402199, APl = 0.59102863, APm = 0.46323815, APs = 0.21414877, ARl = 0.74805754, ARm = 0.62390226, ARmax1 = 0.32788682, ARmax10 = 0.52038383, ARmax100 = 0.55101603, ARs = 0.32568377, box_loss = 0.0, cls_loss = 39.20923, global_step = 0, loss = 39.2865\n",
            "I0510 20:24:56.908696 139834041194368 estimator.py:2066] Saving dict for global step 0: AP = 0.40841353, AP50 = 0.5991094, AP75 = 0.4402199, APl = 0.59102863, APm = 0.46323815, APs = 0.21414877, ARl = 0.74805754, ARm = 0.62390226, ARmax1 = 0.32788682, ARmax10 = 0.52038383, ARmax100 = 0.55101603, ARs = 0.32568377, box_loss = 0.0, cls_loss = 39.20923, global_step = 0, loss = 39.2865\n",
            "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 0: /content/automl/efficientdet/efficientdet-d1/model\n",
            "I0510 20:24:58.433520 139834041194368 estimator.py:2127] Saving 'checkpoint_path' summary for global step 0: /content/automl/efficientdet/efficientdet-d1/model\n",
            "I0510 20:24:58.434658 139834041194368 main.py:351] /content/automl/efficientdet/efficientdet-d1/model has no global step info: stop!\n"
          ]
        }
      ],
      "source": [
        "# Evalute on validation set (takes about 10 mins for efficientdet-d0)\n",
        "!python main.py --mode=eval  \\\n",
        "    --model_name={MODEL}  --model_dir={ckpt_path}  \\\n",
        "    --val_file_pattern=tfrecord/val*  \\\n",
        "    --val_json_file=annotations/instances_val2017.json  \\\n",
        "    --hparams=\"mean_rgb=0.0,stddev_rgb=1.0,scale_range=True\""
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [],
      "name": "Det-AdvProp-tutorial.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}
